package main

import (
	"flag"
	"gxgeek.com/gxgeek/learngo/crawler_distributed/persist/client"
	"gxgeek.com/gxgeek/learngo/crawler/engine"
	"gxgeek.com/gxgeek/learngo/crawler/scheduler"
	"gxgeek.com/gxgeek/learngo/crawler/zhenai/parser"
	"gxgeek.com/gxgeek/learngo/crawler/config"
	worker "gxgeek.com/gxgeek/learngo/crawler_distributed/worker/client"
	"gxgeek.com/gxgeek/learngo/crawler_distributed/rpcsupport"
	"net/rpc"
	"log"
	"errors"
	"strings"
)

var (
	itemSaverHost = flag.String(
		"itemsaver_host", "", "itemsaver host")

	workerHosts = flag.String(
		"worker_hosts", "",
		"worker hosts (comma separated)")
)

//启动参数  -itemsaver_host=:1234 -worker_hosts=:9000
func main()  {

	flag.Parse()
	itemChan, err := client.ItemSaver( *itemSaverHost)
	//itemChan, err := client.ItemSaver( fmt.Sprintf(":%s",config.TestSavePort))
	if err != nil {
		panic(err)
	}

	//processor,err := worker.CreateRequestProcessor()
	//if err!=nil {
	//	panic(err)
	//}

	pool, err := createClientPool(
		strings.Split(*workerHosts, ","))
	if err != nil {
		panic(err)
	}

	processor := worker.CreateProcessor(pool)


	concurrentEngine := engine.ConcurrentEngine{
		Scheduler: &scheduler.QueuedScheduler{},
		WorkerCount:100,
		ItemChan:itemChan,
		RequestProcessor: processor,

	}
	concurrentEngine.Run(engine.Request{
		Url:config.ZHENG_HUN_URL,
		Parser:engine.NewFuncParser(parser.ParseCityList,config.ParseCityList),
	})

	//e := engine.ConcurrentEngine{
	//	Scheduler:        &scheduler.QueuedScheduler{},
	//	WorkerCount:      100,
	//	ItemChan:         itemChan,
	//	Processor: processor,
	//}
	//
	//e.Run(engine.Requests{
	//	Url: "http://www.starter.url.here",
	//	Parser: engine.NewFuncParser(
	//		parser.ParseCityList,
	//		config.ParseCityList),
	//})

}
//创建 client
func createClientPool(
	hosts []string) (chan *rpc.Client, error) {
	var clients []*rpc.Client
	for _, h := range hosts {
		client, err := rpcsupport.NewClient(h)
		if err == nil {
			clients = append(clients, client)
			log.Printf("Worker Client Connected to %s", h)
		} else {
			log.Printf(
				"Error connecting to %s: %v",
				h, err)
		}
	}

	if len(clients) == 0 {
		return nil, errors.New(
			"no connections available")
	}
	out := make(chan *rpc.Client)
	go func() {
		for {
			for _, client := range clients {
				out <- client
			}
		}
	}()
	return out, nil
}
